﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Xml;
using System.Xml.XPath;
using System.Xml.Linq;
using System.Threading;
using System.IO;
using System.Collections.Specialized;
using System.Data.SqlClient;
using DepBank;

namespace TestBench
{
    public partial class frmTestBench : Form
    {
        public frmTestBench()
        {
            InitializeComponent();
        }
        delegate void SetTextCallback(string text);
        delegate void SetValueCallback();



        int senCount = 0;
        int invokeCount_Sentences = 0;
        int fileCount = 0;
        string _strDataDir = @"C:\Data\bnc_rasp_xml";

        string _fileList = @"C:\Data\FileSet.txt";
        string _erroLog = @"C:\Personal\Project\TestBench\Output\";

        List<StringCollection> filePartition = new List<StringCollection>();
        private bool[] _thread_run;
        Semaphore _pool;

        Dictionary<string, byte> RPOSDic = new Dictionary<string, byte>();
        Dictionary<byte, string> RPOSIDDic = new Dictionary<byte, string>();
        Dictionary<string, byte> GRTypeDic = new Dictionary<string, byte>();
        Dictionary<byte, string> GRTypeIDDic = new Dictionary<byte, string>();
        Dictionary<string, byte> SubTypeDic = new Dictionary<string, byte>();
        Dictionary<byte, string> SubTypeIDDic = new Dictionary<byte, string>();
        Dictionary<string, short> WordDic = new Dictionary<string, short>();
        Dictionary<string, short> LemaDic = new Dictionary<string, short>();


        StringCollection poslist = new StringCollection();

        private void btnBNCDataExtract_Click(object sender, EventArgs e)
        {


        }
        private void btnStart_Click(object sender, EventArgs e)
        {
            int numOfThreads = int.Parse(tbxNumOfThreads.Text);
            int semaphoreCount = int.Parse(tbxSemaphore.Text);
            _pool = new Semaphore(0, semaphoreCount);
            filePartition.Clear();
            StringCollection fileSet = new StringCollection();


            InitDictionaries();



            int maxFiles = 0;
            using (StreamReader sr = new StreamReader(_fileList))
            {
                while (!sr.EndOfStream)
                {
                    maxFiles++;
                    string relative = sr.ReadLine();
                    string absolute = String.Empty;
                    absolute = relative.Replace("#", _strDataDir);
                    fileSet.Add(absolute);
                }
            }

            int division = (int)Math.Floor(((decimal)fileSet.Count) / ((decimal)numOfThreads));
            int lastsplit = division + (fileSet.Count % numOfThreads);
            int startIndex = 0;
            for (int i = 0; i < numOfThreads - 1; i++)
            {
                StringCollection partition = new StringCollection();
                startIndex = i * division;
                for (int j = startIndex; j < startIndex + division; j++)
                {
                    partition.Add(fileSet[j]);
                }
                filePartition.Add(partition);
            }

            startIndex = (numOfThreads - 1) * division;
            if (startIndex > 0 || numOfThreads == 1)
            {
                StringCollection lastPartition = new StringCollection();
                for (int i = startIndex; i < startIndex + lastsplit; i++)
                {
                    lastPartition.Add(fileSet[i]);
                }
                filePartition.Add(lastPartition);
            }
            _thread_run = new bool[numOfThreads];
            for (int i = 0; i < numOfThreads; i++)
            {
                _thread_run[i] = true;
                System.Threading.ThreadPool.QueueUserWorkItem(new WaitCallback(File_CreateGR_Bigram), i);
            }
            _pool.Release(semaphoreCount);

        }
        private void File_CreateWordDic(Object s)
        {
            _pool.WaitOne();
            int threadID = (int)s;
            StringCollection fileSet = filePartition[threadID];

            using (BNC_DBDataContext context = new BNC_DBDataContext())
            {


                foreach (string xmlFilePath in fileSet)
                {

                    System.Threading.Interlocked.Increment(ref fileCount);
                    if (!_thread_run[threadID])
                    {
                        DisplayMessage("Thread " + threadID + " returned");
                        _pool.Release();
                        return;
                    }


                    XDocument doc = XDocument.Load(xmlFilePath);

                    short fileID = (from file in context.TBL_Files where file.FileName == xmlFilePath select file.FileID).Single();
                    List<Sentence> sentneceAccumilator = new List<Sentence>();

                    List<WordToken> tokenSet = new List<WordToken>();
                    #region File Process

                    //Process Written Text
                    XElement wtext = doc.XPathSelectElement("/bncDoc/wtext");
                    XElement stext = doc.XPathSelectElement("/bncDoc/stext");
                    if (wtext != null)
                    {
                        #region Written Text

                        var wtext_div = wtext.Elements("div");

                        foreach (var div in wtext_div)
                        {

                            var sentenceSet = div.Descendants("s");
                            foreach (XElement sen in sentenceSet)
                            {
                                Dictionary<int, WordToken> values = CreateWord(sen);
                                tokenSet.AddRange(values.Values);
                            }

                        }
                        wtext = null;
                        #endregion
                    }


                    if (stext != null)
                    {
                        #region Speech Utterance

                        var stext_div = stext.Elements("u");
                        foreach (var div in stext_div)
                        {
                            var sentenceSet = div.Descendants("s");

                            foreach (var sen in sentenceSet)
                            {
                                Dictionary<int, WordToken> values = CreateWord(sen);
                                tokenSet.AddRange(values.Values);
                            }

                        }
                        #endregion
                        stext = null;

                    }

                    #endregion


                    string folder = @"C:\Data\sriout\BNC_GROut\bigramModel";

                    using (StreamWriter wr = new StreamWriter(folder + "\\" + threadID + ".txt", true))
                    {
                        foreach (WordToken tok in tokenSet)
                        {
                            string value = tok.Lema + "\t" + tok.Word + "\t" + RPOSIDDic[tok.RPOS.Value] + "\t" + tok.RPOS.Value.ToString() + "\t" + tok.Affix;
                            //wr.WriteLine(value);
                        }
                    }
                    doc.XPathSelectElement("/bncDoc").Value = "";

                    DisplayFileCount();



                }
                DisplayMessage("Thread " + threadID + " Finished!############");
                _pool.Release();
            }
        }
        private void File_CreateGR(Object s)
        {
            _pool.WaitOne();
            int threadID = (int)s;
            StringCollection fileSet = filePartition[threadID];

            using (BNC_DBDataContext context = new BNC_DBDataContext())
            {

                foreach (string xmlFilePath in fileSet)
                {

                    System.Threading.Interlocked.Increment(ref fileCount);
                    if (!_thread_run[threadID])
                    {
                        DisplayMessage("Thread " + threadID + " returned");
                        _pool.Release();
                        return;
                    }


                    XDocument doc = XDocument.Load(xmlFilePath);

                    short fileID = (from file in context.TBL_Files where file.FileName == xmlFilePath select file.FileID).Single();
                    List<Sentence> sentneceAccumilator = new List<Sentence>();


                    #region File Process

                    //Process Written Text
                    XElement wtext = doc.XPathSelectElement("/bncDoc/wtext");
                    XElement stext = doc.XPathSelectElement("/bncDoc/stext");

                    if (wtext != null)
                    {
                        #region Written Text

                        var wtext_div = wtext.Elements("div");
                        int divisionID = 0;
                        int sentenceID = 0;
                        foreach (var div in wtext_div)
                        {
                            divisionID++;
                            var sentenceSet = div.Descendants("s");


                            foreach (XElement sen in sentenceSet)
                            {
                                sentenceID++;
                                Sentence processedSen = CreateSentence("wtext", fileID, divisionID, sentenceID, sen);
                                sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion
                    }


                    if (stext != null)
                    {
                        #region Speech Utterance


                        var stext_div = stext.Elements("u");
                        int divisionID = 0;
                        int sentenceID = 0;
                        foreach (var div in stext_div)
                        {
                            divisionID++;
                            var sentenceSet = div.Descendants("s");

                            foreach (var sen in sentenceSet)
                            {
                                sentenceID++;
                                Sentence processedSen = CreateSentence("stext", fileID, divisionID, sentenceID, sen);
                                sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion

                    }

                    #endregion

                    string folderPath = @"C:\Data\sriout\BNC_GROut\bigramModel";

                    using (StreamWriter wr = new StreamWriter(folderPath + "\\gr_out_" + threadID + ".txt", true))
                    {
                        {

                            foreach (Sentence sen in sentneceAccumilator)
                            {
                                foreach(GR gg in sen.GRSet){

                                   
                                    if(!String.IsNullOrEmpty(gg.H_Lema)){
                                        gg.H_Lema = gg.H_Lema.ToLower();
                                        if (gg.H_Lema.Length > 25)
                                        {
                                            gg.H_Lema = gg.H_Lema.Substring(0, 25);
                                        }
                                    }
                                    if(!String.IsNullOrEmpty(gg.D_Lema)){
                                        gg.D_Lema = gg.D_Lema.ToLower();
                                        if (gg.D_Lema.Length > 25)
                                        {
                                            gg.D_Lema = gg.D_Lema.Substring(0, 25);
                                        }
                                    }
                                    if (!String.IsNullOrEmpty(gg.SubType) && gg.SubType.Length>10) {
                                        gg.SubType = gg.SubType.Substring(0, 10);
                                    }

                                    if (!String.IsNullOrEmpty(gg.InitGR) && gg.InitGR.Length > 10)
                                    {
                                        gg.InitGR = gg.InitGR.Substring(0, 10);
                                    }

                                    StringBuilder sb=new StringBuilder();
                                    sb.Append(gg.BulkInstFileID);
                                    sb.Append("\t");
                                    sb.Append(gg.BulkInstSenID);
                                    sb.Append("\t");
                                    sb.Append(GRTypeIDDic[gg.GRType_ID]);
                                    sb.Append("\t");
                                    sb.Append(gg.H_Lema);
                                    sb.Append("\t");
                                    if (gg.H_Lema_RPOS.HasValue) { sb.Append(RPOSIDDic[gg.H_Lema_RPOS.Value]); }
                                    sb.Append("\t");
                                    sb.Append(gg.D_Lema);
                                    sb.Append("\t");
                                    if (gg.D_Lema_RPOS.HasValue) { sb.Append(RPOSIDDic[gg.D_Lema_RPOS.Value]); }
                                    sb.Append("\t");
                                    sb.Append(gg.SubType);
                                    sb.Append("\t");
                                    sb.Append(gg.InitGR);

                                    string value = sb.ToString();

                                    wr.WriteLine(value);
                                    
                                    }


                            }


                        }


                        DisplayFileCount();

                    }

                }

                _pool.Release();
            }
        }
        private void File_CreateGR_Bigram(Object s)
        {
            _pool.WaitOne();
            int threadID = (int)s;
            StringCollection fileSet = filePartition[threadID];

            using (BNC_DBDataContext context = new BNC_DBDataContext())
            {

                foreach (string xmlFilePath in fileSet)
                {

                    System.Threading.Interlocked.Increment(ref fileCount);
                    if (!_thread_run[threadID])
                    {
                        DisplayMessage("Thread " + threadID + " returned");
                        _pool.Release();
                        return;
                    }


                    XDocument doc = XDocument.Load(xmlFilePath);

                    short fileID = (from file in context.TBL_Files where file.FileName == xmlFilePath select file.FileID).Single();
                    List<Sentence> sentneceAccumilator = new List<Sentence>();


                    #region File Process

                    //Process Written Text
                    XElement wtext = doc.XPathSelectElement("/bncDoc/wtext");
                    XElement stext = doc.XPathSelectElement("/bncDoc/stext");

                    if (wtext != null)
                    {
                        #region Written Text

                        var wtext_div = wtext.Elements("div");
                        int divisionID = 0;
                        int sentenceID = 0;
                        foreach (var div in wtext_div)
                        {
                            divisionID++;
                            var sentenceSet = div.Descendants("s");


                            foreach (XElement sen in sentenceSet)
                            {
                                sentenceID++;
                                Sentence processedSen = CreateSentence("wtext", fileID, divisionID, sentenceID, sen);
                                sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion
                    }


                    if (stext != null)
                    {
                        #region Speech Utterance


                        var stext_div = stext.Elements("u");
                        int divisionID = 0;
                        int sentenceID = 0;
                        foreach (var div in stext_div)
                        {
                            divisionID++;
                            var sentenceSet = div.Descendants("s");

                            foreach (var sen in sentenceSet)
                            {
                                sentenceID++;
                                Sentence processedSen = CreateSentence("stext", fileID, divisionID, sentenceID, sen);
                                sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion

                    }

                    #endregion

                    string folderPath = @"C:\Data\sriout\BNC_GROut\bigramModel";

                    using (StreamWriter wr = new StreamWriter(folderPath + "\\gr_out_" + threadID + ".txt", true))
                    {
                        {

                            foreach (Sentence sen in sentneceAccumilator)
                            {
                                var H_D=from x in sen.GRSet where (from dep in sen.GRSet select dep.HeadID).Contains(x.DependentID) select x;
                                
                                foreach (var h in H_D) {
                                    var depSet = from dep in sen.GRSet where h.DependentID == dep.HeadID select dep;
                                    GR gr_1 = FormatGR(h);
                                    foreach (var dep in depSet) {
                                        GR gr_2 = FormatGR(dep);

                                        StringBuilder sb = new StringBuilder();
                                        sb.Append(gr_1.BulkInstFileID);
                                        sb.Append("\t");
                                        sb.Append(gr_1.BulkInstSenID);
                                        sb.Append("\t");
                                        sb.Append(GRTypeIDDic[gr_1.GRType_ID] + "_" + GRTypeIDDic[gr_2.GRType_ID]);
                                        sb.Append("\t");
                                        sb.Append(GRTypeIDDic[gr_1.GRType_ID]);
                                        sb.Append("\t");
                                        sb.Append(GRTypeIDDic[gr_2.GRType_ID]);
                                        sb.Append("\t");
                                        sb.Append(gr_1.H_Lema);
                                        sb.Append("\t");
                                        if (gr_1.H_Lema_RPOS.HasValue) { sb.Append(RPOSIDDic[gr_1.H_Lema_RPOS.Value]); }
                                        sb.Append("\t");
                                        sb.Append(gr_1.D_Lema);
                                        sb.Append("\t");
                                        if (gr_1.D_Lema_RPOS.HasValue) { sb.Append(RPOSIDDic[gr_1.D_Lema_RPOS.Value]); }
                                        sb.Append("\t");
                                        sb.Append(gr_2.D_Lema);
                                        sb.Append("\t");
                                        if (gr_2.D_Lema_RPOS.HasValue) { sb.Append(RPOSIDDic[gr_2.D_Lema_RPOS.Value]); }
                                        sb.Append("\t");
                                        sb.Append(gr_1.SubType);
                                        sb.Append("\t");
                                        sb.Append(gr_2.SubType);
                                        sb.Append("\t");
                                        sb.Append(gr_1.InitGR);
                                        sb.Append("\t");
                                        sb.Append(gr_2.InitGR);


                                        string value = sb.ToString();
                                        wr.WriteLine(value);
                                    
                                    }
                                
                                }

                    


                            }


                        }


                        DisplayFileCount();

                    }

                }

                _pool.Release();
            }
        }

        private GR FormatGR(GR pGR) {

            if (!String.IsNullOrEmpty(pGR.H_Lema))
            {
                pGR.H_Lema = pGR.H_Lema.ToLower();
                if (pGR.H_Lema.Length > 25)
                {
                    pGR.H_Lema = pGR.H_Lema.Substring(0, 25);
                }
            }
            if (!String.IsNullOrEmpty(pGR.D_Lema))
            {
                pGR.D_Lema = pGR.D_Lema.ToLower();
                if (pGR.D_Lema.Length > 25)
                {
                    pGR.D_Lema = pGR.D_Lema.Substring(0, 25);
                }
            }
            if (!String.IsNullOrEmpty(pGR.SubType) && pGR.SubType.Length > 10)
            {
                pGR.SubType = pGR.SubType.Substring(0, 10);
            }

            if (!String.IsNullOrEmpty(pGR.InitGR) && pGR.InitGR.Length > 10)
            {
                pGR.InitGR = pGR.InitGR.Substring(0, 10);
            }
            return pGR;
        }
        private void CreateWordDic(Object s)
        {
            _pool.WaitOne();
            int threadID = (int)s;
            StringCollection fileSet = filePartition[threadID];

            using (BNC_DBDataContext context = new BNC_DBDataContext())
            {
                DataTable tb_w = new DataTable();
                tb_w.Columns.Add("Lema", typeof(string));
                tb_w.Columns.Add("Word", typeof(string));
                tb_w.Columns.Add("POS", typeof(string));
                tb_w.Columns.Add("POSID", typeof(short));
                tb_w.Columns.Add("Affix", typeof(string));


                foreach (string xmlFilePath in fileSet)
                {

                    System.Threading.Interlocked.Increment(ref fileCount);
                    if (!_thread_run[threadID])
                    {
                        DisplayMessage("Thread " + threadID + " returned");
                        _pool.Release();
                        return;
                    }


                    XDocument doc = XDocument.Load(xmlFilePath);

                    short fileID = (from file in context.TBL_Files where file.FileName == xmlFilePath select file.FileID).Single();
                    List<Sentence> sentneceAccumilator = new List<Sentence>();

                    List<WordToken> tokenSet = new List<WordToken>();
                    #region File Process

                    //Process Written Text
                    XElement wtext = doc.XPathSelectElement("/bncDoc/wtext");
                    XElement stext = doc.XPathSelectElement("/bncDoc/stext");
                    if (wtext != null)
                    {
                        #region Written Text

                        var wtext_div = wtext.Elements("div");

                        foreach (var div in wtext_div)
                        {

                            var sentenceSet = div.Descendants("s");
                            foreach (XElement sen in sentenceSet)
                            {
                                Dictionary<int, WordToken> values = CreateWord(sen);
                                tokenSet.AddRange(values.Values);
                            }

                        }
                        wtext = null;
                        #endregion
                    }


                    if (stext != null)
                    {
                        #region Speech Utterance

                        var stext_div = stext.Elements("u");
                        foreach (var div in stext_div)
                        {
                            var sentenceSet = div.Descendants("s");

                            foreach (var sen in sentenceSet)
                            {
                                Dictionary<int, WordToken> values = CreateWord(sen);
                                tokenSet.AddRange(values.Values);
                            }

                        }
                        #endregion
                        stext = null;

                    }

                    #endregion



                    #region DB write


                    foreach (WordToken tok in tokenSet)
                    {

                        DataRow row_s = tb_w.NewRow();
                        if (String.IsNullOrEmpty(tok.Lema))
                        {
                            row_s["Lema"] = DBNull.Value;
                        }
                        else
                        {
                            row_s["Lema"] = tok.Lema;
                        }

                        if (String.IsNullOrEmpty(tok.Word))
                        {
                            row_s["Word"] = DBNull.Value;
                        }
                        else
                        {
                            row_s["Word"] = tok.Word;
                        }


                        if (tok.RPOS.HasValue)
                        {
                            row_s["POSID"] = tok.RPOS.Value;
                            row_s["POS"] = RPOSIDDic[tok.RPOS.Value];
                        }
                        else
                        {
                            row_s["POSID"] = DBNull.Value;
                            row_s["POS"] = DBNull.Value;
                        }

                        if (String.IsNullOrEmpty(tok.Affix))
                        {
                            row_s["Affix"] = DBNull.Value;
                        }
                        else
                        {
                            row_s["Affix"] = tok.Affix;
                        }

                        tb_w.Rows.Add(row_s);



                    #endregion


                    }
                    doc.XPathSelectElement("/bncDoc").Value = "";

                    DisplayFileCount();


                    DisplayMessage("Thread " + threadID + " Finished!############");

                }
                WriteWord(tb_w, threadID);
                _pool.Release();
            }
        }
        private void WriteWord(DataTable word, int threadID)
        {

            try
            {
                using (SqlConnection con = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["TestBench.Properties.Settings.BNC_Full"].ConnectionString))
                {

                    if (con.State == ConnectionState.Closed)
                    {
                        con.Open();
                    }
                    using (SqlBulkCopy bc = new SqlBulkCopy(con))
                    {

                        bc.BulkCopyTimeout = 180;
                        bc.ColumnMappings.Add("Lema", "Lema");
                        bc.ColumnMappings.Add("Word", "Word");
                        bc.ColumnMappings.Add("POS", "POS");
                        bc.ColumnMappings.Add("POSID", "POSID");
                        bc.ColumnMappings.Add("Affix", "Affix");
                        bc.DestinationTableName = "TBL_Word";
                        bc.WriteToServer(word);

                        bc.ColumnMappings.Clear();

                        bc.Close();

                    }

                }
            }
            catch (Exception ex333)
            {
                int a = 0;
            }

        }

        private void Process(Object s)
        {
            _pool.WaitOne();
            int threadID = (int)s;
            StringCollection fileSet = filePartition[threadID];

            using (BNC_DBDataContext context = new BNC_DBDataContext())
            {

                foreach (string xmlFilePath in fileSet)
                {

                    System.Threading.Interlocked.Increment(ref fileCount);
                    if (!_thread_run[threadID])
                    {
                        DisplayMessage("Thread " + threadID + " returned");
                        _pool.Release();
                        return;
                    }


                    XDocument doc = XDocument.Load(xmlFilePath);

                    short fileID = (from file in context.TBL_Files where file.FileName == xmlFilePath select file.FileID).Single();
                    List<Sentence> sentneceAccumilator = new List<Sentence>();


                    #region File Process

                    //Process Written Text
                    XElement wtext = doc.XPathSelectElement("/bncDoc/wtext");
                    XElement stext = doc.XPathSelectElement("/bncDoc/stext");

                    if (wtext != null)
                    {
                        #region Written Text

                        var wtext_div = wtext.Elements("div");
                        int divisionID = 0;
                        int sentenceID = 0;
                        foreach (var div in wtext_div)
                        {
                            divisionID++;
                            var sentenceSet = div.Descendants("s");


                            foreach (XElement sen in sentenceSet)
                            {
                                sentenceID++;
                                Sentence processedSen = CreateSentence("wtext", fileID, divisionID, sentenceID, sen);
                                // sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion
                    }


                    if (stext != null)
                    {
                        #region Speech Utterance


                        var stext_div = stext.Elements("u");
                        int divisionID = 0;
                        int sentenceID = 0;
                        foreach (var div in stext_div)
                        {
                            divisionID++;
                            var sentenceSet = div.Descendants("s");

                            foreach (var sen in sentenceSet)
                            {
                                sentenceID++;
                                Sentence processedSen = CreateSentence("stext", fileID, divisionID, sentenceID, sen);
                                //sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion

                    }

                    #endregion



                    #region DB write

                    //DataTable tb_s = new DataTable();
                    //tb_s.Columns.Add("FileID", typeof(short));
                    //tb_s.Columns.Add("DivisionNumber", typeof(int));
                    //tb_s.Columns.Add("SentenceNumber", typeof(int));
                    //tb_s.Columns.Add("Sentence", typeof(string));
                    //tb_s.Columns.Add("GRScore", typeof(double));
                    //tb_s.Columns.Add("Parent", typeof(string));
                    //tb_s.Columns.Add("ParentAttributes", typeof(string));
                    //tb_s.Columns.Add("HierarchyOrder", typeof(string));


                    //DataTable tb_gr_aggregate = new DataTable();
                    //tb_gr_aggregate.Columns.Add("BulkInstFileID", typeof(int));
                    //tb_gr_aggregate.Columns.Add("BulkInstSenID", typeof(int));
                    //tb_gr_aggregate.Columns.Add("GRType", typeof(byte));
                    //tb_gr_aggregate.Columns.Add("Head_Location", typeof(int));
                    //tb_gr_aggregate.Columns.Add("Dependent_Location", typeof(int));
                    //tb_gr_aggregate.Columns.Add("SubType", typeof(string));
                    //tb_gr_aggregate.Columns.Add("InitGR", typeof(string));
                    //tb_gr_aggregate.Columns.Add("HeadVocabID", typeof(int));
                    //tb_gr_aggregate.Columns.Add("DependentVocabID", typeof(int));
                    //tb_gr_aggregate.Columns.Add("H_Lema", typeof(string));
                    //tb_gr_aggregate.Columns.Add("H_Lema_RPOS", typeof(byte));
                    //tb_gr_aggregate.Columns.Add("D_Lema", typeof(string));
                    //tb_gr_aggregate.Columns.Add("D_Lema_RPOS", typeof(byte));
                    //tb_gr_aggregate.Columns.Add("H_Word", typeof(string));
                    //tb_gr_aggregate.Columns.Add("D_Word", typeof(string));
                    //tb_gr_aggregate.Columns.Add("H_HW", typeof(string));
                    //tb_gr_aggregate.Columns.Add("D_HW", typeof(string));


                    //foreach (Sentence sen in sentneceAccumilator)
                    //{

                    //    DataRow row_s = tb_s.NewRow();
                    //    #region Sentence Table
                    //    row_s["DivisionNumber"] = sen.DivisionNumber;
                    //    row_s["SentenceNumber"] = sen.SentenceNumber;
                    //    row_s["FileID"] = fileID;
                    //    if (String.IsNullOrEmpty(sen.SentenceValue))
                    //    {
                    //        row_s["Sentence"] = DBNull.Value;
                    //    }
                    //    else
                    //    {
                    //        row_s["Sentence"] = sen.SentenceValue;
                    //    }

                    //    if (sen.GRScore.HasValue)
                    //    {
                    //        row_s["GRScore"] = sen.GRScore.Value;
                    //    }
                    //    else
                    //    {
                    //        row_s["GRScore"] = DBNull.Value;
                    //    }


                    //    if (String.IsNullOrEmpty(sen.Parent))
                    //    {
                    //        row_s["Parent"] = DBNull.Value;
                    //    }
                    //    else
                    //    {
                    //        row_s["Parent"] = sen.Parent;
                    //    }

                    //    if (String.IsNullOrEmpty(sen.HierarchyOrder))
                    //    {
                    //        row_s["HierarchyOrder"] = DBNull.Value;
                    //    }
                    //    else
                    //    {
                    //        row_s["HierarchyOrder"] = sen.HierarchyOrder;
                    //    }
                    //    #endregion
                    //    tb_s.Rows.Add(row_s);


                    //    foreach (GR gr in sen.GRSet)
                    //    {

                    //        DataRow row_gr = tb_gr_aggregate.NewRow();
                    //        #region GR Table
                    //        row_gr["BulkInstFileID"] = gr.BulkInstFileID;
                    //        row_gr["BulkInstSenID"] = gr.BulkInstSenID;


                    //        if (gr.GRType_ID == 0)
                    //        {
                    //            row_gr["GRType"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["GRType"] = gr.GRType_ID;
                    //        }

                    //        if (gr.HeadID.HasValue)
                    //        {
                    //            row_gr["Head_Location"] = gr.HeadID.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["Head_Location"] = DBNull.Value;
                    //        }

                    //        if (gr.DependentID.HasValue)
                    //        {
                    //            row_gr["Dependent_Location"] = gr.DependentID.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["Dependent_Location"] = DBNull.Value;
                    //        }


                    //        if (String.IsNullOrEmpty(gr.SubType))
                    //        {
                    //            row_gr["SubType"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["SubType"] = gr.SubType;
                    //        }

                    //        if (String.IsNullOrEmpty(gr.InitGR))
                    //        {
                    //            row_gr["InitGR"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["InitGR"] = gr.InitGR;
                    //        }

                    //        if (gr.HeadVocabID.HasValue)
                    //        {
                    //            row_gr["HeadVocabID"] = gr.HeadVocabID.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["HeadVocabID"] = DBNull.Value;
                    //        }

                    //        if (gr.DependentVocabID.HasValue)
                    //        {
                    //            row_gr["DependentVocabID"] = gr.DependentVocabID.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["DependentVocabID"] = DBNull.Value;
                    //        }


                    //        if (String.IsNullOrEmpty(gr.H_Lema))
                    //        {
                    //            row_gr["H_Lema"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["H_Lema"] = gr.H_Lema;
                    //        }

                    //        if (gr.H_Lema_RPOS.HasValue)
                    //        {
                    //            row_gr["H_Lema_RPOS"] = gr.H_Lema_RPOS.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["H_Lema_RPOS"] = DBNull.Value;

                    //        }

                    //        if (String.IsNullOrEmpty(gr.D_Lema))
                    //        {
                    //            row_gr["D_Lema"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["D_Lema"] = gr.D_Lema;
                    //        }

                    //        if (gr.D_Lema_RPOS.HasValue)
                    //        {
                    //            row_gr["D_Lema_RPOS"] = gr.D_Lema_RPOS.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["D_Lema_RPOS"] = DBNull.Value;
                    //        }

                    //        if (String.IsNullOrEmpty(gr.H_Word))
                    //        {
                    //            row_gr["H_Word"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["H_Word"] = gr.H_Word;
                    //        }
                    //        if (String.IsNullOrEmpty(gr.D_Word))
                    //        {
                    //            row_gr["D_Word"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["D_Word"] = gr.D_Word;

                    //        }
                    //        if (String.IsNullOrEmpty(gr.H_HW))
                    //        {
                    //            row_gr["H_HW"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["H_HW"] = gr.H_HW;
                    //        }
                    //        if (String.IsNullOrEmpty(gr.D_HW))
                    //        {
                    //            row_gr["D_HW"] = DBNull.Value;
                    //        }
                    //        else
                    //        {
                    //            row_gr["D_HW"] = gr.D_HW;
                    //        }

                    //        #endregion
                    //        tb_gr_aggregate.Rows.Add(row_gr);

                    //    }


                    //}

                    //WriteDB(tb_gr_aggregate, null, tb_s, threadID, fileID);



                    #endregion

                    DisplayFileCount();

                }





                DisplayMessage("Thread " + threadID + " Finished!############");


            }



            _pool.Release();
        }



        private void WriteDB(DataTable gr, DataTable word, DataTable sen, int threadID, int fileID)
        {
            using (StreamWriter logFile = new StreamWriter(_erroLog + "thread_" + threadID + ".txt"))
            {
                try
                {
                    using (SqlConnection con = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["TestBench.Properties.Settings.BNC_Corpus_TestConnectionString"].ConnectionString))
                    {

                        if (con.State == ConnectionState.Closed)
                        {
                            con.Open();
                        }
                        using (SqlBulkCopy bc = new SqlBulkCopy(con))
                        {

                            bc.BulkCopyTimeout = 180;
                            bc.ColumnMappings.Add("FileID", "FileID");
                            bc.ColumnMappings.Add("DivisionNumber", "DivisionNumber");
                            bc.ColumnMappings.Add("SentenceNumber", "SentenceNumber");
                            bc.ColumnMappings.Add("Sentence", "Sentence");
                            bc.ColumnMappings.Add("GRScore", "GRScore");
                            bc.ColumnMappings.Add("Parent", "Parent");
                            bc.ColumnMappings.Add("ParentAttributes", "ParentAttributes");
                            bc.ColumnMappings.Add("HierarchyOrder", "HierarchyOrder");
                            bc.DestinationTableName = "TBL_Sentence";
                            bc.WriteToServer(sen);

                            bc.ColumnMappings.Clear();



                            bc.ColumnMappings.Add("BulkInstFileID", "BulkInstFileID");
                            bc.ColumnMappings.Add("BulkInstSenID", "BulkInstSenID");
                            bc.ColumnMappings.Add("GRType", "GRType");
                            bc.ColumnMappings.Add("Head_Location", "Head_Location");
                            bc.ColumnMappings.Add("Dependent_Location", "Dependent_Location");
                            bc.ColumnMappings.Add("SubType", "SubType");
                            bc.ColumnMappings.Add("InitGR", "InitGR");
                            bc.ColumnMappings.Add("HeadVocabID", "HeadVocabID");
                            bc.ColumnMappings.Add("DependentVocabID", "DependentVocabID");

                            bc.ColumnMappings.Add("H_Lema", "H_Lema");
                            bc.ColumnMappings.Add("H_Lema_RPOS", "H_Lema_RPOS");
                            bc.ColumnMappings.Add("D_Lema", "D_Lema");
                            bc.ColumnMappings.Add("D_Lema_RPOS", "D_Lema_RPOS");

                            bc.ColumnMappings.Add("H_Word", "H_Word");
                            bc.ColumnMappings.Add("D_Word", "D_Word");
                            bc.ColumnMappings.Add("H_HW", "H_HW");
                            bc.ColumnMappings.Add("D_HW", "D_HW");

                            bc.DestinationTableName = "TBL_GR_Aggregate";

                            bc.WriteToServer(gr);

                            using (BNC_DBDataContext context = new BNC_DBDataContext())
                            {
                                TBL_File dbFile = (from f in context.TBL_Files where f.FileID == fileID select f).Single();
                                dbFile.IsSuccessful = true;
                                context.SubmitChanges();

                            }



                            bc.Close();

                        }

                    }
                }
                catch (Exception ex333)
                {
                    logFile.WriteLine("Error : " + threadID + " file ID " + fileID + " " + ex333.Message);
                }
            }
        }
        private Sentence CreateSentence(string pSenType, int pFileID, int pLocalGenDivisionID, int pLocalGenSentenceID, XElement pSen)
        {

            Sentence resultSentence = new Sentence();
            resultSentence.DivisionNumber = pLocalGenDivisionID;
            resultSentence.SentenceNumber = pLocalGenSentenceID;
            resultSentence.SentenceValue = pSen.Value;
            resultSentence.Parent = pSen.Parent.Name.LocalName;

            DisplaySetenceCount();

            var parent = pSen.Parent;
            string parent_attributes = (from p in parent.Attributes() select p.Name + "=" + p.Value).DefaultIfEmpty<string>().Aggregate<string>((i, j) => i + ";" + j);
            string heirarchy = String.Empty;
            while (!parent.IsEmpty && parent.Name != pSenType)
            {
                heirarchy = parent.Name + ">" + heirarchy;
                parent = parent.Parent;
            }

            resultSentence.HierarchyOrder = heirarchy;

            Dictionary<int, WordToken> wordSet = CreateWord(pSen);
            resultSentence.WordSet = wordSet;
            var grlist = pSen.Element("grlist");
            var grSet = grlist != null ? grlist.Descendants("gr") : null;
            if (grSet != null)
            {

                foreach (XElement gr in grSet)
                {
                    GR singleGR = new GR();
                    string grtypeKey = gr.Attribute("type") != null ? gr.Attribute("type").Value : null;
                    singleGR.GRType_ID = String.IsNullOrEmpty(grtypeKey) ? (byte)0 : GRTypeDic[grtypeKey.Trim()];
                    singleGR.SubType = gr.Attribute("subtype") != null ? gr.Attribute("subtype").Value : null;
                    int headID = -1;
                    singleGR.HeadID = (gr.Attribute("head") != null && int.TryParse(gr.Attribute("head").Value, out headID)) ? headID : (int?)null;
                    int depID = -1;
                    singleGR.DependentID = (gr.Attribute("dep") != null && int.TryParse(gr.Attribute("dep").Value, out depID)) ? depID : (int?)null;
                    singleGR.BulkInstFileID = pFileID;
                    singleGR.BulkInstSenID = pLocalGenSentenceID;
                    if (headID != -1 && wordSet.ContainsKey(headID))
                    {
                        WordToken token = wordSet[headID];
                        singleGR.H_Lema = token.Lema;
                        singleGR.H_Lema_RPOS = token.RPOS;
                        singleGR.H_HW = token.HW;
                        singleGR.H_Word = token.Word;
                    }

                    if (depID != -1 && wordSet.ContainsKey(depID))
                    {
                        WordToken token = wordSet[depID];
                        singleGR.D_Lema = token.Lema;
                        singleGR.D_Lema_RPOS = token.RPOS;
                        singleGR.D_HW = token.HW;
                        singleGR.D_Word = token.Word;
                    }

                    resultSentence.GRSet.Add(singleGR);

                }
                double score = 0;
                resultSentence.GRScore = (grlist.Attribute("score") != null && double.TryParse(grlist.Attribute("score").Value, out score)) ? score : (double?)(null);

            }
            return resultSentence;

        }

        private Dictionary<int, WordToken> CreateWord(XElement sen)
        {
            var tokenSet = sen.Descendants("w").Union(sen.Descendants("c"));
            Dictionary<int, WordToken> result = new Dictionary<int, WordToken>();

            foreach (XElement token in tokenSet)
            {

                string[] tokenIDSet = token.Attribute("n") != null ? token.Attribute("n").Value.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries) : null;
                string rpos = token.Attributes("rpos").Count() > 0 ? token.Attributes("rpos").Single().Value : null;

                if (tokenIDSet != null)
                {
                    if (tokenIDSet.Length > 1 && !String.IsNullOrEmpty(rpos))
                    {

                        string hwList = token.Attributes("hw").Count() > 0 ? token.Attributes("hw").Single().Value : null;
                        string mainWordList = token.Value != null ? token.Value : null;
                        if (!String.IsNullOrEmpty(hwList) && !String.IsNullOrEmpty(mainWordList))
                        {
                            string[] HWsplitWords = hwList.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
                            string[] MainSplitWords = mainWordList.Split(new string[] { "/" }, StringSplitOptions.RemoveEmptyEntries);
                            HWsplitWords = String.Join("#$/#$", HWsplitWords).Split(new string[] { "#$" }, StringSplitOptions.RemoveEmptyEntries);
                            MainSplitWords = String.Join("#$/#$", MainSplitWords).Split(new string[] { "#$" }, StringSplitOptions.RemoveEmptyEntries);
                            string[] rposTags = rpos.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);


                            if (MainSplitWords.Length == tokenIDSet.Length && MainSplitWords.Length == rposTags.Length)
                            {
                                for (int SWIndex = 0; SWIndex < MainSplitWords.Length; SWIndex++)
                                {

                                    int location = int.Parse(tokenIDSet[SWIndex]);
                                    if (!result.ContainsKey(location))
                                    {
                                        WordToken word = new WordToken();
                                        if (HWsplitWords != null && HWsplitWords.Length == MainSplitWords.Length)
                                        {
                                            word.HW = HWsplitWords[SWIndex];
                                        }
                                        word.Word = MainSplitWords[SWIndex];
                                        word.Lema = token.Attributes("lem").Count() > 0 ? token.Attributes("lem").Single().Value : null;
                                        word.Affix = token.Attributes("affix").Count() > 0 ? token.Attributes("affix").Single().Value : null;

                                        string rposKey = rposTags[SWIndex];

                                        word.RPOS = String.IsNullOrEmpty(rposKey) ? (byte?)null : RPOSDic[rposKey.Trim()];
                                        word.WordType = "s";

                                        //string wordKey = word.Lema + "_" + word.RPOS + "_" + word.Word;
                                        //if (!WordDic.Keys.Contains(wordKey))
                                        //{
                                        result.Add(location, word);
                                        // }
                                    }

                                }
                            }
                        }

                    }
                    else
                    {
                        int location;

                        if (tokenIDSet.Length == 1 && int.TryParse(tokenIDSet[0], out location))
                        {
                            if (!result.ContainsKey(location))
                            {
                                WordToken word = new WordToken();
                                word.WordType = token.Parent.Name.LocalName == "mw" ? "m" + token.Name.LocalName[0].ToString() : token.Name.LocalName[0].ToString();
                                string rposKey = token.Attributes("rpos").Count() > 0 ? token.Attributes("rpos").Single().Value : null;

                                word.RPOS = String.IsNullOrEmpty(rposKey) ? (byte)0 : RPOSDic[rposKey];
                                word.HW = token.Attributes("hw").Count() > 0 ? token.Attributes("hw").Single().Value : null;
                                word.Lema = token.Attributes("lem").Count() > 0 ? token.Attributes("lem").Single().Value : null;
                                word.Affix = token.Attributes("affix").Count() > 0 ? token.Attributes("affix").Single().Value : null;
                                word.Word = token.Value;

                                //string wordKey = word.Lema + "_" + word.RPOS + "_" + word.Word;
                                //if (!WordDic.Keys.Contains(wordKey))
                                //{
                                result.Add(location, word);
                                //}
                            }

                        }
                    }
                }
            }
            return result;
        }
        private StringCollection CreateRPOS(XElement sen)
        {
            var tokenSet = sen.Descendants("w").Union(sen.Descendants("c"));

            StringCollection result = new StringCollection();

            foreach (XElement token in tokenSet)
            {

                string[] tokenIDSet = token.Attribute("n") != null ? token.Attribute("n").Value.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries) : null;
                string rpos = token.Attributes("rpos").Count() > 0 ? token.Attributes("rpos").Single().Value : null;

                if (tokenIDSet != null)
                {
                    if (tokenIDSet.Length > 1 && !String.IsNullOrEmpty(rpos))
                    {
                        string[] rposTags = rpos.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries);
                        for (int posTagIndex = 0; posTagIndex < rposTags.Length; posTagIndex++)
                        {
                            string rposKey = rposTags[posTagIndex].Trim();
                            if (!RPOSDic.ContainsKey(rposKey) && !result.Contains(rpos))
                            {
                                result.Add(rposKey);
                            }
                        }
                    }
                    else
                    {
                        int location;
                        if (tokenIDSet.Length == 1 && int.TryParse(tokenIDSet[0], out location))
                        {
                            string rposKey = token.Attributes("rpos").Count() > 0 ? token.Attributes("rpos").Single().Value.Trim() : null;
                            if (!RPOSDic.ContainsKey(rposKey) && !result.Contains(rpos))
                            {
                                result.Add(rposKey);
                            }

                        }
                    }
                }
            }
            return result;
        }

        private void DisplayMessage(string text)
        {
            if (tbxMessageDisplay.InvokeRequired)
            {
                SetTextCallback d = new SetTextCallback(DisplayMessage);
                this.Invoke(d, new object[] { text });
            }
            else
            {
                if (tbxMessageDisplay.Text.Length > 2000)
                {
                    tbxMessageDisplay.Text = "";
                }
                tbxMessageDisplay.AppendText(text);
                tbxMessageDisplay.AppendText("\r\n");
                tbxMessageDisplay.ScrollToCaret();
            }
        }
        private void DisplaySetenceCount()
        {
            invokeCount_Sentences++;
            System.Threading.Interlocked.Increment(ref senCount);
            if (lblSentenceCount.InvokeRequired)
            {
                if (invokeCount_Sentences > 1000)
                {
                    SetValueCallback d = new SetValueCallback(DisplaySetenceCount);
                    this.Invoke(d);
                }
            }
            else
            {
                invokeCount_Sentences = 0;
                lblSentenceCount.Text = senCount.ToString();
                lblFileCount.Text = fileCount.ToString();
            }
        }

        private void DisplayFileCount()
        {

            invokeCount_Sentences++;
            System.Threading.Interlocked.Increment(ref senCount);
            if (lblFileCount.InvokeRequired)
            {
                SetValueCallback d = new SetValueCallback(DisplayFileCount);
                this.Invoke(d);
            }
            else
            {
                lblFileCount.Text = fileCount.ToString();
            }
        }


        private void InitDictionaries()
        {
            using (DepBank_DBDataContext context = new DepBank_DBDataContext())
            {

                RPOSDic.Clear();
                RPOSIDDic.Clear();
                GRTypeDic.Clear();
                GRTypeIDDic.Clear();
                WordDic.Clear();
                SubTypeDic.Clear();
                SubTypeIDDic.Clear();
                LemaDic.Clear();
                var varRPOSSet = from r in context.TBL_RPOs select r;
                foreach (var rpos in varRPOSSet)
                {
                    RPOSDic.Add(rpos.RPOS.Trim(), rpos.RPOSID);
                    RPOSIDDic.Add(rpos.RPOSID, rpos.RPOS.Trim());

                }

                var varGRTypeSet = from gr in context.TBL_GRTypes select gr;
                foreach (var grval in varGRTypeSet)
                {
                    GRTypeDic.Add(grval.GRType.Trim(), grval.GRTypeID);
                    GRTypeIDDic.Add(grval.GRTypeID, grval.GRType.Trim());
                }

                var subTypeSet = from s in context.TBL_SubTypes select s;
                foreach (var subType in subTypeSet)
                {
                    SubTypeDic.Add(subType.SubType.Trim(), subType.SubTypeID);
                    SubTypeIDDic.Add(subType.SubTypeID, subType.SubType.Trim());
                }
                var lemaSet = from l in context.TBL_LemaDics select l;
                foreach (var l in lemaSet)
                {
                    LemaDic.Add(l.Lema.Trim(), l.LemaID);
                }

                var wordset = from w in context.TBL_WordDics select w;
                foreach (var word in wordset)
                {
                    string key = word.Lema + "_" + word.POS_ID + "_" + word.Word;
                    WordDic.Add(key, word.WordID);
                }


            }
        }
        private void Test(Object ss)
        {
            System.Threading.Thread.Sleep(2000);
        }
        private void btnStop_Click(object sender, EventArgs e)
        {
            if (_thread_run != null)
            {

                for (int i = 0; i < _thread_run.Length; i++)
                {
                    _thread_run[i] = false;
                }
            }
        }

        private void btnGC_Click(object sender, EventArgs e)
        {
            GC.Collect();
            GC.WaitForPendingFinalizers();
        }

        private void btnSetSemaphore_Click(object sender, EventArgs e)
        {
            int releaseCount = int.Parse(tbxSetSemaphore.Text);
            _pool.Release(releaseCount);


        }

        private void btnDepBankLoad_Click(object sender, EventArgs e)
        {
            string _depBankFile = @"C:\Data\DepBank\sents-hld.gr";
            char evaluationSetType = 'd';

            Dictionary<string, string> grTemplateDic = GetGRDic();
            using (DepBank_DBDataContext context = new DepBank_DBDataContext())
            {
                GRTypeDic.Clear();

                var dbGRSet = from g in context.TBL_GRTypes select g;
                foreach (var grtype in dbGRSet)
                {
                    GRTypeDic.Add(grtype.GRType.Trim(), grtype.GRTypeID);
                }

                using (StreamReader reader = new StreamReader(_depBankFile))
                {
                    string line = reader.ReadLine();
                    line = reader.ReadLine();
                    do
                    {

                        int senID = 0;
                        if (int.TryParse(line, out senID))
                        {

                            while (!string.IsNullOrEmpty(line))
                            {
                                TBL_GoldStandard_GR gr = new TBL_GoldStandard_GR();
                                gr.EvaluationSetType = evaluationSetType;
                                line = reader.ReadLine();
                                if (!String.IsNullOrEmpty(line) && line.Trim().Length > 0)
                                {
                                    string[] grTokens = line.Split(new string[] { " ", "(", ")" }, StringSplitOptions.RemoveEmptyEntries);
                                    gr.GRType = grTokens[0];
                                    gr.GRTypeID = GRTypeDic[grTokens[0]];
                                    gr.SentenceID = senID;
                                    string template = grTemplateDic[grTokens[0]];
                                    if (grTokens.Length - 1 == template.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries).Length)
                                    {

                                        switch (template)
                                        {
                                            case "S H D":
                                                {
                                                    gr.SubType = grTokens[1];
                                                    gr.H_Word = grTokens[2];
                                                    gr.D_Word = grTokens[3];
                                                    break;
                                                }
                                            case "H D":
                                                {
                                                    gr.H_Word = grTokens[1];
                                                    gr.D_Word = grTokens[2];
                                                    break;
                                                }
                                            case "H D I":
                                                {
                                                    gr.H_Word = grTokens[1];
                                                    gr.D_Word = grTokens[2];
                                                    gr.InitGR = grTokens[3];
                                                    break;
                                                }
                                            case "H":
                                                {
                                                    gr.H_Word = grTokens[1];
                                                    break;
                                                }
                                        }

                                    }

                                    int headID = -1;
                                    int depID = -1;
                                    if (!String.IsNullOrEmpty(gr.H_Word))
                                    {
                                        TBL_Vocab vocab;
                                        var word = from w in context.TBL_Vocabs where w.Word == gr.H_Word select w;
                                        if (word.Count() == 0)
                                        {
                                            vocab = new TBL_Vocab();
                                            vocab.Word = gr.H_Word;
                                            context.TBL_Vocabs.InsertOnSubmit(vocab);
                                            context.SubmitChanges();
                                        }
                                        else
                                        {
                                            vocab = word.Single();
                                        }
                                        gr.H_Word_VocabID = vocab.VocabID;
                                    }
                                    else
                                    {
                                        //No template match
                                        int k = 0;
                                    }

                                    if (!String.IsNullOrEmpty(gr.D_Word))
                                    {
                                        TBL_Vocab vocab;
                                        var word = from w in context.TBL_Vocabs where w.Word == gr.D_Word select w;
                                        if (word.Count() == 0)
                                        {
                                            vocab = new TBL_Vocab();
                                            vocab.Word = gr.D_Word;
                                            context.TBL_Vocabs.InsertOnSubmit(vocab);
                                            context.SubmitChanges();
                                        }
                                        else
                                        {
                                            vocab = word.Single();
                                        }
                                        gr.D_Word_VocabID = vocab.VocabID;
                                    }
                                }
                                context.TBL_GoldStandard_GRs.InsertOnSubmit(gr);
                                context.SubmitChanges();

                            }
                            line = reader.ReadLine();
                        }
                    } while ((!String.IsNullOrEmpty(line)));
                }
            }
        }

        private Dictionary<string, string> GetGRDic()
        {
            Dictionary<string, string> parameterCount = new Dictionary<string, string>();
            parameterCount.Add("dependent", "S H D");
            parameterCount.Add("mod","S H D");
            parameterCount.Add("subj_dobj", "H D");
            parameterCount.Add("clausal", "H D");

            parameterCount.Add("arg", "S H D");
            parameterCount.Add("arg_mod", "S H D");
            parameterCount.Add("aux", "H D");
            parameterCount.Add("ccomp", "S H D");
            parameterCount.Add("cmod", "S H D");
            parameterCount.Add("comp", "H D");
            parameterCount.Add("conj", "H D");
            parameterCount.Add("csubj", "H D I");
            parameterCount.Add("det", "H D");
            parameterCount.Add("dobj", "H D");
            parameterCount.Add("iobj", "H D");
            parameterCount.Add("ncmod", "S H D");
            parameterCount.Add("ncsubj", "H D I");
            parameterCount.Add("obj", "H D");
            parameterCount.Add("obj2", "H D");
            parameterCount.Add("passive", "H");
            parameterCount.Add("pcomp", "H D");
            parameterCount.Add("pmod", "H D");
            parameterCount.Add("ta", "S H D");
            parameterCount.Add("xcomp", "S H D");
            parameterCount.Add("xmod", "S H D");
            parameterCount.Add("xsubj", "H D I");
            return parameterCount;
        }

        private void btnRASPLoad_Click(object sender, EventArgs e)
        {
            string devSet = @"C:\VirtualMachines\VMSharedFolder\140_n400_lema.outg";
            string testSet = @"C:\VirtualMachines\VMSharedFolder\560_n400_lema.outg";
            string failing = @"C:\VirtualMachines\VMSharedFolder\140faling_10_lema.outg";
            string raspLoadFile = testSet;
            char evaluationSetType = 't';
            bool isLema = true;

            XDocument doc = XDocument.Load(raspLoadFile);
            XElement head = doc.XPathSelectElement("/rasp");
            var senList = head.Elements("sentence");

            DataTable rasp_gr_table = new DataTable();
            #region rasp gr table
            rasp_gr_table.Columns.Add("EvaluationSetType", typeof(char));
            rasp_gr_table.Columns.Add("SentenceID", typeof(int));
            rasp_gr_table.Columns.Add("ParseID", typeof(int));
            rasp_gr_table.Columns.Add("GRType", typeof(string));
            rasp_gr_table.Columns.Add("GRTypeID", typeof(byte));
            rasp_gr_table.Columns.Add("H_Word", typeof(string));
            rasp_gr_table.Columns.Add("H_LemaID", typeof(int));
            rasp_gr_table.Columns.Add("H_Lema", typeof(string));
            rasp_gr_table.Columns.Add("H_Lema_Location", typeof(int));
            rasp_gr_table.Columns.Add("H_Lema_RPOS", typeof(string));
            rasp_gr_table.Columns.Add("H_Lema_RPOSID", typeof(short));
            rasp_gr_table.Columns.Add("D_Word", typeof(string));
            rasp_gr_table.Columns.Add("D_LemaID", typeof(int));
            rasp_gr_table.Columns.Add("D_Lema", typeof(string));
            rasp_gr_table.Columns.Add("D_Lema_Location", typeof(int));
            rasp_gr_table.Columns.Add("D_Lema_RPOS", typeof(string));
            rasp_gr_table.Columns.Add("D_Lema_RPOSID", typeof(short));

            rasp_gr_table.Columns.Add("SubType", typeof(string));
            rasp_gr_table.Columns.Add("InitGR", typeof(string));
            rasp_gr_table.Columns.Add("H_Word_DicID", typeof(short));
            rasp_gr_table.Columns.Add("D_Word_DicID", typeof(short));

            #endregion

            using (DepBank_DBDataContext context = new DepBank_DBDataContext())
            {
                InitDictionaries();
                StringCollection sc = new StringCollection();
                foreach (XElement sen in senList)
                {
                    int senNumber = int.Parse(sen.Attribute("num").Value);

                    XElement nbestParse = sen.XPathSelectElement("nbest-parses");
                    if (nbestParse != null)
                    {
                        //int numOfParses = int.Parse(nbestParse.Attribute("num").Value);
                        //TBL_Sen tempSen = new TBL_Sen() { EvaluationSetType = evaluationSetType, SentenceID = senNumber, NumOfParses = numOfParses };
                        //var dbSen = from s in context.TBL_Sens
                        //            where s.EvaluationSetType == tempSen.EvaluationSetType && s.SentenceID == tempSen.SentenceID
                        //            select s;
                        //if (dbSen.Count() == 1)
                        //{
                        //    TBL_Sen currentSen = dbSen.Single<TBL_Sen>();
                        //    currentSen.NumOfParses = tempSen.NumOfParses;
                        //    context.SubmitChanges();
                        //}


                        // context.TBL_Sens.InsertOnSubmit(tblSen);


                        Dictionary<int, TBL_Lema> tokenList = new Dictionary<int, TBL_Lema>();
                        var llist = from l in context.TBL_Lemas where l.SentenceID == senNumber && l.EvaluationSetType == evaluationSetType select l;
                        foreach (TBL_Lema lema in llist)
                        {
                            tokenList.Add(lema.Num, lema);
                        }


                        var parSet = sen.XPathSelectElement("nbest-parses").Elements("parse-set");

                        foreach (XElement parse in parSet)
                        {
                            int parseID = int.Parse(parse.Attribute("pnum").Value);
                            float score = float.Parse(parse.Attribute("score").Value);

                            TBL_Parse tblParse = new TBL_Parse() { EvaluationSetType = evaluationSetType, SentenceID = senNumber, ParseID = parseID, Score = (double)score };
                            var dbParse = from p in context.TBL_Parses where p.EvaluationSetType == tblParse.EvaluationSetType && p.SentenceID == tblParse.SentenceID && p.ParseID == tblParse.ParseID select p;
                            if (dbParse.Count() > 0)
                            {
                                TBL_Parse currentParse = dbParse.Single<TBL_Parse>();
                                currentParse.Score = tblParse.Score;
                                context.SubmitChanges();
                            }
                            else
                            {
                                context.TBL_Parses.InsertOnSubmit(tblParse);
                            }

                            var grlist = parse.Elements("gr-list").Elements("gr");
                            foreach (XElement gr in grlist)
                            {

                                DataRow row = rasp_gr_table.NewRow();
                                //TBL_RASP_GR raspgr = new TBL_RASP_GR();
                                //raspgr.EvaluationSetType = evaluationSetType;
                                //raspgr.SentenceID = senNumber;
                                //raspgr.ParseID = parseID;
                                //raspgr.GRType = gr.Attribute("type").Value.Trim();
                                //raspgr.GRTypeID = GRTypeDic[raspgr.GRType];
                                //raspgr.SubType = gr.Attribute("subtype") == null ? null : gr.Attribute("subtype").Value.Trim();
                                //raspgr.InitGR = gr.Attribute("init") == null ? null : gr.Attribute("init").Value.Trim();

                                row["EvaluationSetType"] = evaluationSetType;
                                row["SentenceID"] = senNumber;
                                row["ParseID"] = parseID;
                                row["GRType"] = gr.Attribute("type").Value.Trim();
                                row["GRTypeID"] = GRTypeDic[gr.Attribute("type").Value.Trim()];
                                if (gr.Attribute("subtype") == null) row["SubType"] = DBNull.Value; else row["SubType"] = gr.Attribute("subtype").Value.Trim();
                                if (gr.Attribute("init") == null) row["InitGR"] = DBNull.Value; else row["InitGR"] = gr.Attribute("init").Value.Trim();


                                int headID = -1;
                                if (gr.Attribute("head") != null && int.TryParse(gr.Attribute("head").Value, out headID))
                                {
                                    //raspgr.H_Lema = tokenList[headID].Lema;
                                    //raspgr.H_Word = tokenList[headID].Word;
                                    //raspgr.H_LemaID = tokenList[headID].LemaID;
                                    //raspgr.H_Lema_RPOS = tokenList[headID].POS;
                                    //raspgr.H_Lema_Location = headID;
                                    row["H_Word"] = tokenList[headID].Word;
                                    row["H_LemaID"] = tokenList[headID].LemaID;
                                    row["H_Lema"] = tokenList[headID].Lema;
                                    row["H_Lema_RPOS"] = tokenList[headID].POS;
                                    row["H_Lema_Location"] = headID;

                                    //string h_key = String.Empty;
                                    //if (!String.IsNullOrEmpty(raspgr.H_Lema) && !String.IsNullOrEmpty(raspgr.H_Lema_RPOS) && !String.IsNullOrEmpty(raspgr.H_Word))
                                    //{
                                    //    string rposID = RPOSDic[raspgr.H_Lema_RPOS.Trim()].ToString();
                                    //    h_key = raspgr.H_Lema.Trim() + "_" + rposID + "_" + raspgr.H_Word.Trim();
                                    //}
                                    //if (WordDic.ContainsKey(h_key))
                                    //{
                                    //    raspgr.H_Word_DicID = WordDic[h_key];
                                    //}

                                    string h_key = String.Empty;
                                    if (!String.IsNullOrEmpty(tokenList[headID].Lema) && !String.IsNullOrEmpty(tokenList[headID].POS) && !String.IsNullOrEmpty(tokenList[headID].Word))
                                    {
                                        string rposID = RPOSDic[tokenList[headID].POS.Trim()].ToString();
                                        row["H_Lema_RPOSID"] = RPOSDic[tokenList[headID].POS.Trim()];
                                        h_key = (tokenList[headID].Lema.Trim() + "_" + rposID + "_" + tokenList[headID].Word.Trim()).ToLower();
                                        if (WordDic.ContainsKey(h_key))
                                        {
                                            row["H_Word_DicID"] = WordDic[h_key];
                                        }
                                        else
                                        {
                                            //DisplayMessage("Missing word 444");
                                            //if (!sc.Contains(h_key))
                                            //{
                                            //    sc.Add(h_key);
                                            //}
                                        }
                                    }


                                }
                                else
                                {
                                    // DisplayMessage("Head missing");
                                }

                                int depID = -1;
                                if (gr.Attribute("dep") != null && int.TryParse(gr.Attribute("dep").Value, out depID))
                                {
                                    //raspgr.D_Lema = tokenList[depID].Lema;
                                    //raspgr.D_Word = tokenList[depID].Word;
                                    //raspgr.D_LemaID = tokenList[depID].LemaID;
                                    //raspgr.D_Lema_RPOS = tokenList[depID].POS;
                                    //raspgr.D_Lema_Location = depID;

                                    row["D_Word"] = tokenList[depID].Word;
                                    row["D_LemaID"] = tokenList[depID].LemaID;
                                    row["D_Lema"] = tokenList[depID].Lema;
                                    row["D_Lema_RPOS"] = tokenList[depID].POS;
                                    row["D_Lema_Location"] = depID;


                                    //string d_key = String.Empty;
                                    //if (!String.IsNullOrEmpty(raspgr.D_Lema) && !String.IsNullOrEmpty(raspgr.D_Lema_RPOS) && !String.IsNullOrEmpty(raspgr.D_Word))
                                    //{
                                    //    string rposID = RPOSDic[raspgr.D_Lema_RPOS.Trim()].ToString();
                                    //    d_key = raspgr.D_Lema.Trim() + "_" + rposID + "_" + raspgr.D_Word.Trim();
                                    //}

                                    //if (WordDic.ContainsKey(d_key))
                                    //{
                                    //    raspgr.D_Word_DicID = WordDic[d_key];
                                    //}

                                    string d_key = String.Empty;
                                    if (!String.IsNullOrEmpty(tokenList[depID].Lema) && !String.IsNullOrEmpty(tokenList[depID].POS) && !String.IsNullOrEmpty(tokenList[depID].Word))
                                    {
                                        string rposID = RPOSDic[tokenList[depID].POS.Trim()].ToString();
                                        row["D_Lema_RPOSID"] = RPOSDic[tokenList[depID].POS.Trim()];
                                        d_key = (tokenList[depID].Lema.Trim() + "_" + rposID + "_" + tokenList[depID].Word.Trim()).ToLower();
                                    }
                                    if (WordDic.ContainsKey(d_key))
                                    {
                                        row["D_Word_DicID"] = WordDic[d_key];
                                    }
                                    else
                                    {
                                        //if (!sc.Contains(d_key)) {
                                        //    sc.Add(d_key);
                                        //}
                                        // DisplayMessage("Missing word 444");
                                    }


                                }

                                //context.TBL_RASP_GRs.InsertOnSubmit(raspgr);
                                rasp_gr_table.Rows.Add(row);
                            }

                        }
                        //WriteRASPGR(rasp_gr_table);
                        rasp_gr_table.Rows.Clear();
                        //context.SubmitChanges();
                        nbestParse = null;
                        GC.Collect();
                        GC.WaitForPendingFinalizers();
                    }
                    else
                    {
                        DisplayMessage("Error in sentence " + senNumber);
                    }
                }

                //for (int i = 0; i < sc.Count; i++) {
                //    DisplayMessage(sc[i]);
                //}

            }
        }

        private void WriteRASPGR(DataTable tbl_fast_load)
        {

            try
            {
                using (SqlConnection con = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["TestBench.Properties.Settings.DepBankConnectionString"].ConnectionString))
                {

                    if (con.State == ConnectionState.Closed)
                    {
                        con.Open();
                    }
                    using (SqlBulkCopy bc = new SqlBulkCopy(con))
                    {

                        bc.BulkCopyTimeout = 180;

                        bc.ColumnMappings.Add("EvaluationSetType", "EvaluationSetType");
                        bc.ColumnMappings.Add("SentenceID", "SentenceID");
                        bc.ColumnMappings.Add("ParseID", "ParseID");
                        bc.ColumnMappings.Add("GRType", "GRType");
                        bc.ColumnMappings.Add("GRTypeID", "GRTypeID");
                        bc.ColumnMappings.Add("H_Word", "H_Word");
                        bc.ColumnMappings.Add("H_LemaID", "H_LemaID");
                        bc.ColumnMappings.Add("H_Lema", "H_Lema");
                        bc.ColumnMappings.Add("H_Lema_Location", "H_Lema_Location");
                        bc.ColumnMappings.Add("H_Lema_RPOS", "H_Lema_RPOS");
                        bc.ColumnMappings.Add("H_Lema_RPOSID", "H_Lema_RPOSID");
                        bc.ColumnMappings.Add("D_Word", "D_Word");
                        bc.ColumnMappings.Add("D_LemaID", "D_LemaID");
                        bc.ColumnMappings.Add("D_Lema", "D_Lema");
                        bc.ColumnMappings.Add("D_Lema_Location", "D_Lema_Location");
                        bc.ColumnMappings.Add("D_Lema_RPOS", "D_Lema_RPOS");
                        bc.ColumnMappings.Add("D_Lema_RPOSID", "D_Lema_RPOSID");
                        bc.ColumnMappings.Add("SubType", "SubType");
                        bc.ColumnMappings.Add("InitGR", "InitGR");
                        bc.ColumnMappings.Add("H_Word_DicID", "H_Word_DicID");
                        bc.ColumnMappings.Add("D_Word_DicID", "D_Word_DicID");

                        bc.DestinationTableName = "TBL_RASP_GR";
                        bc.WriteToServer(tbl_fast_load);

                        bc.Close();

                    }

                }
            }
            catch (Exception ex333)
            {
                int a = 0;
            }

        }

        private void btnRASPLoad_Weighted_Click(object sender, EventArgs e)
        {
            string devSet = @"C:\VirtualMachines\VMSharedFolder\raspout_weightedGR\560_n10_missingGR.xml";
            string testSet = @"";
            string failing = @"";
            string raspLoadFile = devSet;
            char evaluationSetType = 't';
            bool isLema = false;

            XDocument doc = XDocument.Load(raspLoadFile);
            XElement head = doc.XPathSelectElement("/rasp");
            var senList = head.Elements("sentence");

            DataTable rasp_gr_table_w = new DataTable();
            #region rasp gr table
            rasp_gr_table_w.Columns.Add("EvaluationSetType", typeof(char));
            rasp_gr_table_w.Columns.Add("SentenceID", typeof(int));
            rasp_gr_table_w.Columns.Add("ParseID", typeof(int));
            rasp_gr_table_w.Columns.Add("Weight", typeof(float));
            rasp_gr_table_w.Columns.Add("GRType", typeof(string));
            rasp_gr_table_w.Columns.Add("GRTypeID", typeof(byte));
            rasp_gr_table_w.Columns.Add("H_Word", typeof(string));
            rasp_gr_table_w.Columns.Add("H_LemaID", typeof(int));
            rasp_gr_table_w.Columns.Add("H_Lema", typeof(string));
            rasp_gr_table_w.Columns.Add("H_Lema_Location", typeof(int));
            rasp_gr_table_w.Columns.Add("H_Lema_RPOS", typeof(string));
            rasp_gr_table_w.Columns.Add("H_Lema_RPOSID", typeof(short));
            rasp_gr_table_w.Columns.Add("D_Word", typeof(string));
            rasp_gr_table_w.Columns.Add("D_LemaID", typeof(int));
            rasp_gr_table_w.Columns.Add("D_Lema", typeof(string));
            rasp_gr_table_w.Columns.Add("D_Lema_Location", typeof(int));
            rasp_gr_table_w.Columns.Add("D_Lema_RPOS", typeof(string));
            rasp_gr_table_w.Columns.Add("D_Lema_RPOSID", typeof(short));
            rasp_gr_table_w.Columns.Add("SubType", typeof(string));
            rasp_gr_table_w.Columns.Add("InitGR", typeof(string));
            rasp_gr_table_w.Columns.Add("H_Word_DicID", typeof(short));
            rasp_gr_table_w.Columns.Add("D_Word_DicID", typeof(short));

            #endregion

            using (DepBank_DBDataContext context = new DepBank_DBDataContext())
            {
                InitDictionaries();
                StringCollection sc = new StringCollection();
                foreach (XElement sen in senList)
                {
                    int senNumber = int.Parse(sen.Attribute("num").Value);

                    XElement nbestParse = sen.XPathSelectElement("nbest-parses");
                    if (nbestParse != null)
                    {
                        //int numOfParses = int.Parse(nbestParse.Attribute("num").Value);
                        //TBL_Sen tempSen = new TBL_Sen() { EvaluationSetType = evaluationSetType, SentenceID = senNumber, NumOfParses = numOfParses };
                        //var dbSen = from s in context.TBL_Sens
                        //            where s.EvaluationSetType == tempSen.EvaluationSetType && s.SentenceID == tempSen.SentenceID
                        //            select s;
                        //if (dbSen.Count() == 1)
                        //{
                        //    TBL_Sen currentSen = dbSen.Single<TBL_Sen>();
                        //    currentSen.NumOfParses = tempSen.NumOfParses;
                        //    context.SubmitChanges();
                        //}


                        // context.TBL_Sens.InsertOnSubmit(tblSen);


                        Dictionary<int, TBL_Lema> tokenList = new Dictionary<int, TBL_Lema>();
                        var llist = from l in context.TBL_Lemas where l.SentenceID == senNumber && l.EvaluationSetType == evaluationSetType select l;
                        foreach (TBL_Lema lema in llist)
                        {
                            tokenList.Add(lema.Num, lema);
                        }


                        var parSet = sen.XPathSelectElement("nbest-parses").Elements("parse-set");

                        foreach (XElement parse in parSet)
                        {
                            int parseID = int.Parse(parse.Attribute("pnum").Value);
                            float score = float.Parse(parse.Attribute("score").Value);

                            TBL_Parse tblParse = new TBL_Parse() { EvaluationSetType = evaluationSetType, SentenceID = senNumber, ParseID = parseID, Score = (double)score };
                            var dbParse = from p in context.TBL_Parses where p.EvaluationSetType == tblParse.EvaluationSetType && p.SentenceID == tblParse.SentenceID && p.ParseID == tblParse.ParseID select p;
                            if (dbParse.Count() > 0)
                            {
                                TBL_Parse currentParse = dbParse.Single<TBL_Parse>();
                                currentParse.Score = tblParse.Score;
                                context.SubmitChanges();
                            }
                            else
                            {
                                context.TBL_Parses.InsertOnSubmit(tblParse);
                            }
                        }

                        var grlist = sen.XPathSelectElement("weighted").Elements("gr");
                        foreach (XElement gr in grlist)
                        {

                            DataRow row = rasp_gr_table_w.NewRow();
             
                            row["EvaluationSetType"] = evaluationSetType;
                            row["SentenceID"] = senNumber;
                            //row["ParseID"] = parseID; no parse ID
                            row["Weight"] = float.Parse(gr.Element("gr-weight").Attribute("weight").Value);
                            row["GRType"] = gr.Attribute("type").Value.Trim();
                            row["GRTypeID"] = GRTypeDic[gr.Attribute("type").Value.Trim()];
                            if (gr.Attribute("subtype") == null) row["SubType"] = DBNull.Value; else row["SubType"] = gr.Attribute("subtype").Value.Trim();
                            if (gr.Attribute("init") == null) row["InitGR"] = DBNull.Value; else row["InitGR"] = gr.Attribute("init").Value.Trim();


                            int headID = -1;
                            if (gr.Attribute("head") != null && int.TryParse(gr.Attribute("head").Value, out headID))
                            {
                
                                row["H_Word"] = tokenList[headID].Word;
                                row["H_LemaID"] = tokenList[headID].LemaID;
                                row["H_Lema"] = tokenList[headID].Lema;
                                row["H_Lema_RPOS"] = tokenList[headID].POS;
                                row["H_Lema_Location"] = headID;

      
                                string h_key = String.Empty;
                                if (!String.IsNullOrEmpty(tokenList[headID].Lema) && !String.IsNullOrEmpty(tokenList[headID].POS) && !String.IsNullOrEmpty(tokenList[headID].Word))
                                {
                                    string rposID = RPOSDic[tokenList[headID].POS.Trim()].ToString();
                                    row["H_Lema_RPOSID"] = RPOSDic[tokenList[headID].POS.Trim()];
                                    h_key = (tokenList[headID].Lema.Trim() + "_" + rposID + "_" + tokenList[headID].Word.Trim()).ToLower();
                                    if (WordDic.ContainsKey(h_key))
                                    {
                                        row["H_Word_DicID"] = WordDic[h_key];
                                    }
                                    else
                                    {
                   
                                    }
                                }


                            }
                            else
                            {
                                // DisplayMessage("Head missing");
                            }

                            int depID = -1;
                            if (gr.Attribute("dep") != null && int.TryParse(gr.Attribute("dep").Value, out depID))
                            {

                                row["D_Word"] = tokenList[depID].Word;
                                row["D_LemaID"] = tokenList[depID].LemaID;
                                row["D_Lema"] = tokenList[depID].Lema;
                                row["D_Lema_RPOS"] = tokenList[depID].POS;
                                row["D_Lema_Location"] = depID;

                                string d_key = String.Empty;
                                if (!String.IsNullOrEmpty(tokenList[depID].Lema) && !String.IsNullOrEmpty(tokenList[depID].POS) && !String.IsNullOrEmpty(tokenList[depID].Word))
                                {
                                    string rposID = RPOSDic[tokenList[depID].POS.Trim()].ToString();
                                    row["D_Lema_RPOSID"] = RPOSDic[tokenList[depID].POS.Trim()];
                                    d_key = (tokenList[depID].Lema.Trim() + "_" + rposID + "_" + tokenList[depID].Word.Trim()).ToLower();
                                }
                                if (WordDic.ContainsKey(d_key))
                                {
                                    row["D_Word_DicID"] = WordDic[d_key];
                                }
                                else
                                {
                 
                                }


                            }

                            rasp_gr_table_w.Rows.Add(row);
                        }


                        WriteRASPGR_Weighted(rasp_gr_table_w);
                        rasp_gr_table_w.Rows.Clear();

                        nbestParse = null;
                        GC.Collect();
                        GC.WaitForPendingFinalizers();
                    }
                    else
                    {
                        DisplayMessage("Error in sentence " + senNumber);
                    }
                }


            }
        }

        private void WriteRASPGR_Weighted(DataTable tbl_fast_load)
        {

            try
            {
                using (SqlConnection con = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["TestBench.Properties.Settings.DepBankConnectionString"].ConnectionString))
                {

                    if (con.State == ConnectionState.Closed)
                    {
                        con.Open();
                    }
                    using (SqlBulkCopy bc = new SqlBulkCopy(con))
                    {

                        bc.BulkCopyTimeout = 180;

                        bc.ColumnMappings.Add("EvaluationSetType", "EvaluationSetType");
                        bc.ColumnMappings.Add("SentenceID", "SentenceID");
                        bc.ColumnMappings.Add("ParseID", "ParseID");
                        bc.ColumnMappings.Add("Weight", "Weight");
                        bc.ColumnMappings.Add("GRType", "GRType");
                        bc.ColumnMappings.Add("GRTypeID", "GRTypeID");
                        bc.ColumnMappings.Add("H_Word", "H_Word");
                        bc.ColumnMappings.Add("H_LemaID", "H_LemaID");
                        bc.ColumnMappings.Add("H_Lema", "H_Lema");
                        bc.ColumnMappings.Add("H_Lema_Location", "H_Lema_Location");
                        bc.ColumnMappings.Add("H_Lema_RPOS", "H_Lema_RPOS");
                        bc.ColumnMappings.Add("H_Lema_RPOSID", "H_Lema_RPOSID");
                        bc.ColumnMappings.Add("D_Word", "D_Word");
                        bc.ColumnMappings.Add("D_LemaID", "D_LemaID");
                        bc.ColumnMappings.Add("D_Lema", "D_Lema");
                        bc.ColumnMappings.Add("D_Lema_Location", "D_Lema_Location");
                        bc.ColumnMappings.Add("D_Lema_RPOS", "D_Lema_RPOS");
                        bc.ColumnMappings.Add("D_Lema_RPOSID", "D_Lema_RPOSID");
                        bc.ColumnMappings.Add("SubType", "SubType");
                        bc.ColumnMappings.Add("InitGR", "InitGR");
                        bc.ColumnMappings.Add("H_Word_DicID", "H_Word_DicID");
                        bc.ColumnMappings.Add("D_Word_DicID", "D_Word_DicID");

                        bc.DestinationTableName = "TBL_RASP_GR_Weighted";
                        bc.WriteToServer(tbl_fast_load);

                        bc.Close();

                    }

                }
            }
            catch (Exception ex333)
            {
                int a = 0;
            }

        }

        
        
        private void btnParseExtract_Click(object sender, EventArgs e)
        {
            string file = @"C:\VirtualMachines\VMSharedFolder\tsri\560.out";
            using (StreamReader r = new StreamReader(file))
            {
                using (DepBank_DBDataContext context = new DepBank_DBDataContext())
                {
                    string value = r.ReadLine().Trim();
                    int senID = 1;
                    while (!r.EndOfStream)
                    {
                        TBL_Sen sen = new TBL_Sen();
                        sen.SentenceID = senID;
                        sen.RASP_Out = value;
                        sen.EvaluationSetType = 't';
                        context.TBL_Sens.InsertOnSubmit(sen);
                        senID++;
                        string temp = r.ReadLine();
                        while (!String.IsNullOrEmpty(temp) && !r.EndOfStream)
                        {
                            temp = r.ReadLine();
                            if (!String.IsNullOrEmpty(temp))
                            {
                                temp = temp.Trim();
                            }
                        }
                        while (String.IsNullOrEmpty(temp) && !r.EndOfStream)
                        {
                            temp = r.ReadLine();
                            if (!String.IsNullOrEmpty(temp))
                            {
                                temp = temp.Trim();
                            }
                        }
                        value = temp.Trim();
                    }
                    context.SubmitChanges();

                }



            }
        }

        private void btnFastLoad_Click(object sender, EventArgs e)
        {
            int numOfThreads = int.Parse(tbxNumOfThreads.Text);
            int semaphoreCount = int.Parse(tbxSemaphore.Text);
            _pool = new Semaphore(0, semaphoreCount);
            filePartition.Clear();
            StringCollection fileSet = new StringCollection();


            InitDictionaries();


            int maxFiles = 0;
            using (StreamReader sr = new StreamReader(_fileList))
            {
                while (!sr.EndOfStream)
                {

                    string relative = sr.ReadLine();
                    string absolute = String.Empty;
                    absolute = relative.Replace("#", _strDataDir);
                    fileSet.Add(absolute);
                }
            }

            int division = (int)Math.Floor(((decimal)fileSet.Count) / ((decimal)numOfThreads));
            int lastsplit = division + (fileSet.Count % numOfThreads);
            int startIndex = 0;
            for (int i = 0; i < numOfThreads - 1; i++)
            {
                StringCollection partition = new StringCollection();
                startIndex = i * division;
                for (int j = startIndex; j < startIndex + division; j++)
                {
                    partition.Add(fileSet[j]);
                }
                filePartition.Add(partition);
            }

            startIndex = (numOfThreads - 1) * division;
            if (startIndex > 0 || numOfThreads == 1)
            {
                StringCollection lastPartition = new StringCollection();
                for (int i = startIndex; i < startIndex + lastsplit; i++)
                {
                    lastPartition.Add(fileSet[i]);
                }
                filePartition.Add(lastPartition);
            }
            _thread_run = new bool[numOfThreads];
            for (int i = 0; i < numOfThreads; i++)
            {
                _thread_run[i] = true;
                System.Threading.ThreadPool.QueueUserWorkItem(new WaitCallback(FastLoad), i);
            }
            _pool.Release(semaphoreCount);
        }
        private void FastLoad(Object s)
        {
            _pool.WaitOne();
            int threadID = (int)s;
            StringCollection fileSet = filePartition[threadID];

            using (BNC_DBDataContext context = new BNC_DBDataContext())
            {

                foreach (string xmlFilePath in fileSet)
                {

                    System.Threading.Interlocked.Increment(ref fileCount);
                    if (!_thread_run[threadID])
                    {
                        DisplayMessage("Thread " + threadID + " returned");
                        _pool.Release();
                        return;
                    }

                    XDocument doc = XDocument.Load(xmlFilePath);

                    List<Sentence> sentneceAccumilator = new List<Sentence>();


                    #region File Process

                    //Process Written Text
                    XElement wtext = doc.XPathSelectElement("/bncDoc/wtext");
                    XElement stext = doc.XPathSelectElement("/bncDoc/stext");

                    if (wtext != null)
                    {
                        #region Written Text

                        var wtext_div = wtext.Elements("div");

                        foreach (var div in wtext_div)
                        {
                            var sentenceSet = div.Descendants("s");

                            foreach (XElement sen in sentenceSet)
                            {
                                Sentence processedSen = FastLoadCreateSentence(sen);
                                sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion
                    }


                    if (stext != null)
                    {
                        #region Speech Utterance


                        var stext_div = stext.Elements("u");
                        foreach (var div in stext_div)
                        {
                            var sentenceSet = div.Descendants("s");
                            foreach (var sen in sentenceSet)
                            {
                                Sentence processedSen = FastLoadCreateSentence(sen);
                                sentneceAccumilator.Add(processedSen);
                            }

                        }
                        #endregion

                    }

                    #endregion

                    #region DB write


                    DataTable tb_fast_load = new DataTable();


                    tb_fast_load.Columns.Add("GRType_ID", typeof(byte));
                    tb_fast_load.Columns.Add("H_Lema_ID", typeof(short));
                    tb_fast_load.Columns.Add("H_RPOS_ID", typeof(byte));
                    tb_fast_load.Columns.Add("H_Word_ID", typeof(short));
                    tb_fast_load.Columns.Add("D_Lema_ID", typeof(short));
                    tb_fast_load.Columns.Add("D_RPOS_ID", typeof(byte));
                    tb_fast_load.Columns.Add("D_Word_ID", typeof(short));
                    tb_fast_load.Columns.Add("SubType_ID", typeof(byte));
                    tb_fast_load.Columns.Add("GRType", typeof(string));
                    tb_fast_load.Columns.Add("H_Lema", typeof(string));
                    tb_fast_load.Columns.Add("H_RPOS", typeof(string));
                    tb_fast_load.Columns.Add("D_Lema", typeof(string));
                    tb_fast_load.Columns.Add("D_RPOS", typeof(string));
                    tb_fast_load.Columns.Add("SubType", typeof(string));
                    tb_fast_load.Columns.Add("InitGR", typeof(string));

                    foreach (Sentence sen in sentneceAccumilator)
                    {
                        foreach (GR gr in sen.GRSet)
                        {

                            DataRow row_gr = tb_fast_load.NewRow();
                            #region GR Table

                            if (gr.GRType_ID == 0)
                            {
                                row_gr["GRType_ID"] = DBNull.Value;
                                row_gr["GRType"] = DBNull.Value;

                            }
                            else
                            {
                                row_gr["GRType_ID"] = gr.GRType_ID;
                                row_gr["GRType"] = GRTypeIDDic[gr.GRType_ID];

                            }


                            if (String.IsNullOrEmpty(gr.SubType))
                            {
                                row_gr["SubType"] = DBNull.Value;
                                row_gr["SubType_ID"] = DBNull.Value;
                            }
                            else
                            {
                                row_gr["SubType"] = gr.SubType;
                                row_gr["SubType_ID"] = SubTypeDic[gr.SubType.Trim()];
                            }

                            if (String.IsNullOrEmpty(gr.InitGR))
                            {
                                row_gr["InitGR"] = DBNull.Value;
                            }
                            else
                            {
                                row_gr["InitGR"] = gr.InitGR;
                            }


                            if (String.IsNullOrEmpty(gr.H_Lema) || !gr.H_Lema_RPOS.HasValue || String.IsNullOrEmpty(gr.H_Word))
                            {
                                row_gr["H_Word_ID"] = DBNull.Value;
                                row_gr["H_Lema_ID"] = DBNull.Value;
                                row_gr["H_RPOS_ID"] = DBNull.Value;
                                row_gr["H_Lema"] = DBNull.Value;
                                row_gr["H_RPOS"] = DBNull.Value;
                            }
                            else
                            {
                                string h_key = gr.H_Lema.Trim() + "_" + gr.H_Lema_RPOS.Value + "_" + gr.H_Word.Trim();
                                row_gr["H_Word_ID"] = WordDic[h_key];
                                row_gr["H_Lema_ID"] = LemaDic[gr.H_Lema.Trim()];
                                row_gr["H_RPOS_ID"] = gr.H_Lema_RPOS.Value;
                                row_gr["H_Lema"] = gr.H_Lema.Trim();
                                row_gr["H_RPOS"] = RPOSIDDic[gr.H_Lema_RPOS.Value];
                            }


                            if (String.IsNullOrEmpty(gr.D_Lema) || !gr.D_Lema_RPOS.HasValue || String.IsNullOrEmpty(gr.D_Word))
                            {
                                row_gr["D_Word_ID"] = DBNull.Value;
                                row_gr["D_Lema_ID"] = DBNull.Value;
                                row_gr["D_RPOS_ID"] = DBNull.Value;
                                row_gr["D_Lema"] = DBNull.Value;
                                row_gr["D_RPOS"] = DBNull.Value;
                            }
                            else
                            {
                                string d_key = gr.D_Lema.Trim() + "_" + gr.D_Lema_RPOS.Value + "_" + gr.D_Word.Trim();
                                row_gr["D_Word_ID"] = WordDic[d_key];
                                row_gr["D_Lema_ID"] = LemaDic[gr.D_Lema.Trim()];
                                row_gr["D_RPOS_ID"] = gr.D_Lema_RPOS.Value;
                                row_gr["D_Lema"] = gr.D_Lema.Trim();
                                row_gr["D_RPOS"] = RPOSIDDic[gr.D_Lema_RPOS.Value];
                            }



                            #endregion
                            tb_fast_load.Rows.Add(row_gr);

                        }


                    }

                    FastLoadWriteDB(threadID, tb_fast_load);

                    DisplayFileCount();

                    #endregion


                }





                DisplayMessage("Thread " + threadID + " Finished!############");


            }


            _pool.Release();
        }

        private Sentence FastLoadCreateSentence(XElement pSen)
        {

            Sentence resultSentence = new Sentence();

            DisplaySetenceCount();

            Dictionary<int, WordToken> wordSet = CreateWord(pSen);

            resultSentence.WordSet = wordSet;
            var grlist = pSen.Element("grlist");
            var grSet = grlist != null ? grlist.Descendants("gr") : null;
            if (grSet != null)
            {

                foreach (XElement gr in grSet)
                {
                    GR singleGR = new GR();
                    string grtypeKey = gr.Attribute("type") != null ? gr.Attribute("type").Value : null;
                    singleGR.GRType_ID = String.IsNullOrEmpty(grtypeKey) ? (byte)0 : GRTypeDic[grtypeKey.Trim()];
                    singleGR.SubType = gr.Attribute("subtype") != null ? gr.Attribute("subtype").Value : null;
                    int headID = -1;
                    singleGR.HeadID = (gr.Attribute("head") != null && int.TryParse(gr.Attribute("head").Value, out headID)) ? headID : (int?)null;
                    int depID = -1;
                    singleGR.DependentID = (gr.Attribute("dep") != null && int.TryParse(gr.Attribute("dep").Value, out depID)) ? depID : (int?)null;

                    bool isHeadValid = false;
                    bool isDepValid = false;

                    if (headID != -1 && wordSet.ContainsKey(headID))
                    {
                        WordToken token = wordSet[headID];
                        singleGR.H_Lema = String.IsNullOrEmpty(token.Lema) ? null : token.Lema.ToLower();
                        singleGR.H_Lema_RPOS = token.RPOS;
                        singleGR.H_HW = String.IsNullOrEmpty(token.HW) ? null : token.HW.ToLower();
                        singleGR.H_Word = String.IsNullOrEmpty(token.Word) ? null : token.Word.ToLower();


                        if (!String.IsNullOrEmpty(singleGR.H_Lema) && singleGR.H_Lema_RPOS.HasValue && !String.IsNullOrEmpty(singleGR.H_Word))
                        {
                            string h_key = singleGR.H_Lema.Trim() + "_" + singleGR.H_Lema_RPOS.Value + "_" + singleGR.H_Word.Trim();
                            isHeadValid = WordDic.Keys.Contains(h_key);
                        }
                    }

                    if (depID != -1 && wordSet.ContainsKey(depID))
                    {
                        WordToken token = wordSet[depID];
                        singleGR.D_Lema = String.IsNullOrEmpty(token.Lema) ? null : token.Lema.ToLower();
                        singleGR.D_Lema_RPOS = token.RPOS;
                        singleGR.D_HW = String.IsNullOrEmpty(token.HW) ? null : token.HW.ToLower();
                        singleGR.D_Word = String.IsNullOrEmpty(token.Word) ? null : token.Word.ToLower();

                        if (!String.IsNullOrEmpty(singleGR.D_Lema) && singleGR.D_Lema_RPOS.HasValue && !String.IsNullOrEmpty(singleGR.D_Word))
                        {
                            string d_key = singleGR.D_Lema.Trim() + "_" + singleGR.D_Lema_RPOS.Value + "_" + singleGR.D_Word.Trim();
                            isDepValid = WordDic.Keys.Contains(d_key);
                        }
                    }

                    if ((depID == -1 && isHeadValid) || (isHeadValid && isDepValid))
                    {
                        resultSentence.GRSet.Add(singleGR);
                    }

                }

            }

            return resultSentence;

        }
        private void FastLoadWriteDB(int threadID, DataTable tbl_fast_load)
        {
            using (StreamWriter logFile = new StreamWriter(_erroLog + "thread_" + threadID + ".txt"))
            {
                try
                {
                    using (SqlConnection con = new SqlConnection(System.Configuration.ConfigurationManager.ConnectionStrings["TestBench.Properties.Settings.BNC_Corpus_TestConnectionString"].ConnectionString))
                    {

                        if (con.State == ConnectionState.Closed)
                        {
                            con.Open();
                        }
                        using (SqlBulkCopy bc = new SqlBulkCopy(con))
                        {

                            bc.BulkCopyTimeout = 180;


                            bc.ColumnMappings.Add("GRType_ID", "GRType_ID");
                            bc.ColumnMappings.Add("H_Lema_ID", "H_Lema_ID");
                            bc.ColumnMappings.Add("H_RPOS_ID", "H_RPOS_ID");
                            bc.ColumnMappings.Add("H_Word_ID", "H_Word_ID");
                            bc.ColumnMappings.Add("D_Lema_ID", "D_Lema_ID");
                            bc.ColumnMappings.Add("D_RPOS_ID", "D_RPOS_ID");
                            bc.ColumnMappings.Add("D_Word_ID", "D_Word_ID");
                            bc.ColumnMappings.Add("SubType_ID", "SubType_ID");
                            bc.ColumnMappings.Add("GRType", "GRType");
                            bc.ColumnMappings.Add("H_Lema", "H_Lema");
                            bc.ColumnMappings.Add("H_RPOS", "H_RPOS");
                            bc.ColumnMappings.Add("D_Lema", "D_Lema");
                            bc.ColumnMappings.Add("D_RPOS", "D_RPOS");
                            bc.ColumnMappings.Add("SubType", "SubType");
                            bc.ColumnMappings.Add("InitGR", "InitGR");

                            bc.DestinationTableName = "TBL_FastLoad";
                            bc.WriteToServer(tbl_fast_load);

                            bc.Close();

                        }

                    }
                }
                catch (Exception ex333)
                {
                    logFile.WriteLine("Error : " + threadID + " " + ex333.Message);
                }
            }
        }



        private void btnAddRoot_Click(object sender, EventArgs e)
        {
            string devSet = @"C:\VirtualMachines\VMSharedFolder\140_n400_word.out";
            string testSet = @"C:\VirtualMachines\VMSharedFolder\560_n400_word.out";
            string failing = @"C:\VirtualMachines\VMSharedFolder\560_failing10_word.out";
            string raspLoadFile = testSet;

            string value = null;
            using (StreamReader reader = new StreamReader(raspLoadFile))
            {
                value = reader.ReadToEnd();
                value = "<rasp>" + value + "</rasp>";
                reader.Close();
            }

            using (StreamWriter r = new StreamWriter(raspLoadFile + "g"))
            {
                r.Write(value);
            }
        }

        private void btnLemaLoad_Click(object sender, EventArgs e)
        {
            string devSet = @"C:\VirtualMachines\VMSharedFolder\140_n400_word.outg";
            string testSet = @"C:\VirtualMachines\VMSharedFolder\560_n400_word.outg";
            string failing = @"C:\VirtualMachines\VMSharedFolder\560_failing10_word.outg";

            string raspLoadFile = testSet;
            char evaluationSetType = 't';
            bool isLemaLoad = false;


            XDocument doc = XDocument.Load(raspLoadFile);
            XElement head = doc.XPathSelectElement("/rasp");
            var senList = head.Elements("sentence");


            using (DepBank_DBDataContext context = new DepBank_DBDataContext())
            {
                InitDictionaries();

                foreach (XElement sen in senList)
                {
                    int senNumber = int.Parse(sen.Attribute("num").Value);

                    XElement nbestParse = sen.XPathSelectElement("nbest-parses");
                    if (nbestParse != null)
                    {
                        int numOfParses = int.Parse(nbestParse.Attribute("num").Value);
                        TBL_Sen tempSen = new TBL_Sen() { EvaluationSetType = evaluationSetType, SentenceID = senNumber, NumOfParses = numOfParses };
                        var dbSen = from s in context.TBL_Sens
                                    where s.EvaluationSetType == tempSen.EvaluationSetType && s.SentenceID == tempSen.SentenceID
                                    select s;
                        if (isLemaLoad && dbSen.Count() == 1)
                        {
                            TBL_Sen currentSen = dbSen.Single<TBL_Sen>();
                            currentSen.NumOfParses = tempSen.NumOfParses;
                            context.SubmitChanges();
                        }
                        //context.TBL_Sens.InsertOnSubmit(tblSen);


                        var lemaList = sen.XPathSelectElement("lemma-list").Elements("lemma");


                        foreach (XElement lema in lemaList)
                        {
                            int num = int.Parse(lema.Attribute("num").Value);
                            int wnum = int.Parse(lema.Attribute("wnum").Value);

                            TBL_Lema token = new TBL_Lema();
                            token.EvaluationSetType = evaluationSetType;
                            token.Num = num;
                            token.Wnum = wnum;
                            token.SentenceID = senNumber;

                            if (isLemaLoad)
                            {
                                token.Lema = lema.Attribute("lem").Value;
                            }

                            token.POS = lema.Attribute("pos").Value;
                            token.Affix = lema.Attribute("affix") == null ? null : lema.Attribute("affix").Value;
                            if (isLemaLoad)
                            {
                                context.TBL_Lemas.InsertOnSubmit(token);
                            }
                            else
                            {
                                TBL_Lema currentLema = (from l in context.TBL_Lemas where l.EvaluationSetType == evaluationSetType && l.SentenceID == senNumber && l.Wnum == wnum && l.Num == num select l).Single();
                                currentLema.Word = lema.Attribute("lem").Value;
                            }
                        }

                        context.SubmitChanges();

                    }
                }
            }
        }

        public class WordToken
        {
            public string HW;
            public string Lema;
            public byte? RPOS;
            public string WordType;
            public string Word;
            public string Affix;
        }

        public class GR
        {

            public byte GRType_ID;
            public string SubType;
            public int? HeadID;
            public int? DependentID;
            public int BulkInstFileID;
            public int BulkInstSenID;
            public string H_Lema;
            public byte? H_Lema_RPOS;
            public string H_HW;
            public string H_Word;
            public string D_Lema;
            public byte? D_Lema_RPOS;
            public string D_HW;
            public string D_Word;
            public string InitGR;
            public int? HeadVocabID;
            public int? DependentVocabID;
        }

        public class Sentence
        {
            public List<GR> GRSet = new List<GR>();
            public Dictionary<int, WordToken> WordSet = new Dictionary<int, WordToken>();
            public int DivisionNumber;
            public int SentenceNumber;
            public string SentenceValue;
            public string Parent;
            public string HierarchyOrder;
            public double? GRScore;
        }

        private void btn_Serialize_Click(object sender, EventArgs e)
        {
            using(BNC_FullDataContext context=new BNC_FullDataContext()){
                context.CommandTimeout = 600;
               TBL_Word[] wordSet=(from w in context.TBL_Words select w).ToArray();
               System.Collections.Hashtable dic=new System.Collections.Hashtable();
               foreach(TBL_Word w in wordSet){
                string key=w.Lema+"_"+w.Word+"_"+w.POS+"_"+w.Affix;
                if (!dic.ContainsKey(key))
                {
                    dic.Add(key, w.WordID);
                }
               }
                using(FileStream fs=new FileStream(@"C:\Data\sriout\BNC_WordOut\wordDic.ser",FileMode.CreateNew)){
                System.Runtime.Serialization.Formatters.Binary.BinaryFormatter bf=new System.Runtime.Serialization.Formatters.Binary.BinaryFormatter();
                bf.Serialize(fs,dic);
                fs.Flush();
               }

            }
        }

        private void btn_DepBank_POSLemaLoad_Click(object sender, EventArgs e)
        {
            string file = @"C:\VirtualMachines\VMSharedFolder\depbanksen\560";
            char evaluationSetType = 't';
            bool isLemaLoad = false;
            using(DepBank_DBDataContext context=new DepBank_DBDataContext()){
                int sentenceID=0;
                using (StreamReader sr = new StreamReader(file))
                {
                    while (!sr.EndOfStream)
                    {
                        string sen = sr.ReadLine();
                        sentenceID++;
                        string[] tokSet = sen.Split(new string[] { " ", "\t" }, StringSplitOptions.RemoveEmptyEntries);
                        for (int i = 1; i < tokSet.Length; i++)
                        {
                            string tok = tokSet[i];
                           
                            DepBank.TBL_DepBank_Sen newTok = new TBL_DepBank_Sen();
                            newTok.EvaluationSetType = evaluationSetType;
                            newTok.SentenceID = sentenceID;
                            newTok.Location = i;
                            if (isLemaLoad)
                            {
                                string[] parts = tok.Split(new string[] { "_", "+" }, StringSplitOptions.RemoveEmptyEntries);
                                if (parts.Length == 3)
                                {
                                    newTok.Lema = parts[0].ToLower();
                                    newTok.Affix = parts[1];
                                    newTok.POS = parts[2];
                                }
                                else if (parts.Length == 2)
                                {
                                    newTok.Lema = parts[0].ToLower();
                                    newTok.POS = parts[1];
                                }
                                context.TBL_DepBank_Sens.InsertOnSubmit(newTok);
                            }
                            else {
                                string[] parts = tok.Split(new string[] { "_"}, StringSplitOptions.RemoveEmptyEntries);
                                var modTok = (from tt in context.TBL_DepBank_Sens where tt.EvaluationSetType == evaluationSetType && tt.SentenceID == sentenceID && tt.Location == i select tt).Single();
                                if (parts.Length == 2)
                                {
                                    modTok.Word = parts[0];
                                    context.SubmitChanges();
                                }
                            }
                            
                        }
                        context.SubmitChanges();
                    }
                }
       
            
            }
        }
    }
}
